#!/bin/bash
# ========================================
# 自动部署：Kubernetes 节点 Ready 监控 & 三步重启脚本
# 全部使用北京时间 (CST) | 阈值：30 分钟 | 每 30 分钟检查一次
# ========================================

set -e

# ---------------- 配置区（北京时间）----------------
SCRIPT_PATH="/opt/k8s-restart-monitor.sh"
LOG_PATH="/var/log/k8s-restart-monitor.log"
CRON_TIME="*/30 * * * *"           # 每30分钟（北京时间）
TZ='Asia/Shanghai'                 # 北京时间
CRON_USER="root"                   # 使用 root 执行

# 确保目录存在
sudo mkdir -p /opt /var/log

# ---------------- 主脚本内容（阈值 30 分钟）----------------
cat <<'EOF' | sudo tee $SCRIPT_PATH > /dev/null
#!/bin/bash
# ========================================
# 节点 Ready 状态 < 30 分钟 则三步重启 Namespace
# 全部使用北京时间 (CST)
# ========================================
export TZ='Asia/Shanghai'  # 强制使用北京时间
CURRENT_EPOCH=$(date +%s)
THRESHOLD_SECONDS=$((30 * 60))  # 30 分钟
RED='\033[31m'
GREEN='\033[32m'
YELLOW='\033[33m'
BLUE='\033[34m'
NC='\033[0m'
log() { echo -e "${BLUE}[INFO]${NC} $*"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
error() { echo -e "${RED}[ERROR]${NC} $*"; }
success(){ echo -e "${GREEN}[SUCCESS]${NC} $*"; }

STEP1=("iam" "cac" "dmc" "emc" "lmc" "eoc" "gmc" "omc" "bossiam")
STEP2=("aam" "abt" "atdm" "atmc" "audc" "im" "package" "uibot")
STEP3=("tbds" "ut" "bm-opsca" "bm-schsc" "bmbasc" "bmbmsc" "bmcsc" "bmd-bmsc-001" "bmd-dmsc-001" "bmd-dmsc-be-gateway" "bmd-dmsc-be-iam" "bmd-dmsc-be-semantic" "bmd-dmsc-fe-main" "bmd-imsc-001" "bmd-mosc-001" "bmd-opsc-001" "bmd-qmsc-001" "bmdbasc001" "bmdirsc001" "bmdpisc001" "bmdposc001" "bmdsdsc001" "bmiisc" "bmimsc" "bmirsc" "bmmisc" "bmmosc" "bmommsc" "bmopsc" "bmpisc" "bmposc" "bmpsc" "bmqmsc" "bmsdsc" "bmtssc" "bmusc" "bmwcisc" "bmwcsc" "bmwhsc")

check_nodes_and_trigger() {
    log "正在检查节点 Ready 状态（北京时间）..."
    local trigger_restart=0
    local alert_nodes=""
    while IFS=$'\t' read -r NODE_NAME LAST_TRANSITION_TIME; do
        [[ -z "$NODE_NAME" || -z "$LAST_TRANSITION_TIME" ]] && continue
        TRANSITION_EPOCH=$(date -u --date="$LAST_TRANSITION_TIME" +%s 2>/dev/null)
        [[ -z "$TRANSITION_EPOCH" ]] && { error "$NODE_NAME: 时间解析失败"; continue; }
        DIFF_SECONDS=$((CURRENT_EPOCH - TRANSITION_EPOCH))
        DIFF_MINUTES=$(echo "scale=1; $DIFF_SECONDS / 60" | bc -l 2>/dev/null || echo "0")
        TRANSITION_CST=$(date -d "@$TRANSITION_EPOCH" '+%Y-%m-%d %H:%M:%S CST')
        echo -e " $NODE_NAME → $TRANSITION_CST → ${YELLOW}${DIFF_MINUTES}m${NC}"
        if (( $(echo "$DIFF_SECONDS < $THRESHOLD_SECONDS" | bc -l) )); then
            alert_nodes+="$NODE_NAME(${DIFF_MINUTES}m) "
            trigger_restart=1
        fi
    done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].lastTransitionTime}{"\n"}{end}')
    echo
    if [[ $trigger_restart -eq 1 ]]; then
        warn "检测到节点最近变更 Ready 状态 (< 30 分钟): $alert_nodes"
        warn "即将执行三步重启流程..."
        return 0
    else
        success "所有节点 Ready 状态 ≥ 30 分钟，无需重启。"
        return 1
    fi
}

restart_namespace() {
    local ns=$1
    if [[ -z "$ns" ]]; then
        error "namespace 名称不能为空"
        return 1
    fi
    log "正在获取 namespace '$ns' 下的所有 Deployment..."
    local deployments=$(kubectl get deployment -n "$ns" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)
    if [[ -z "$deployments" ]]; then
        warn "namespace '$ns' 中未找到任何 Deployment，跳过重启。"
        return 0
    fi
    local count=0
    while IFS= read -r dep; do
        [[ -z "$dep" ]] && continue
        log "重启 Deployment: $ns/$dep"
        if kubectl rollout restart deployment "$dep" -n "$ns" >/dev/null 2>&1; then
            success "已触发 $ns/$dep 重启"
            ((count++))
        else
            error "重启失败: $ns/$dep"
        fi
    done <<< "$deployments"
    log "namespace '$ns' 共重启 $count 个 Deployment"
}

run_step() {
    local step=$1
    shift
    local resources=("$@")
    log "=== 第 $step 步：重启 ${#resources[@]} 个 namespace ==="
    for res in "${resources[@]}"; do
        restart_namespace "$res"
    done
}

main() {
    echo "==========================================="
    echo "Kubernetes 节点 Ready 监控 & 重启脚本"
    echo "当前时间（北京时间）: $(date '+%Y-%m-%d %H:%M:%S %Z')"
    echo "告警条件: 存在节点 Ready 变更 < 30 分钟"
    echo "==========================================="
    echo
    if ! check_nodes_and_trigger; then
        echo
        success "无需重启，脚本结束。"
        exit 0
    fi
    echo
    run_step 1 "${STEP1[@]}"
    warn "第1步完成，等待 3 分钟..."
    sleep 180
    run_step 2 "${STEP2[@]}"
    warn "第2步完成，等待 3 分钟..."
    sleep 180
    run_step 3 "${STEP3[@]}"
    echo
    success "全部重启完成！请观察服务状态。"
}

main
EOF

# ---------------- 权限设置 ----------------
sudo chmod +x $SCRIPT_PATH
sudo touch $LOG_PATH
sudo chmod 644 $LOG_PATH

# ---------------- 安全添加 crontab 函数 ----------------
add_cron_job() {
    local cron_job="$1"
    local user="$2"
    local temp_cron="/tmp/crontab.tmp.$$.cron"

    echo "正在为用户 [$user] 添加 crontab 任务..."
    echo "任务: $cron_job"

    # 导出当前 crontab
    if sudo -u "$user" crontab -l > "$temp_cron" 2>/dev/null; then
        echo "[$user] 当前 crontab 已导出"
    else
        echo "[$user] 无 crontab，创建新文件"
        > "$temp_cron"
    fi

    # 删除旧任务
    grep -v -F "$SCRIPT_PATH" "$temp_cron" > "${temp_cron}.clean" 2>/dev/null || true
    mv "${temp_cron}.clean" "$temp_cron"

    # 添加新任务
    echo "$cron_job" >> "$temp_cron"

    # 写入 crontab
    if sudo -u "$user" crontab "$temp_cron"; then
        echo "[$user] crontab 添加成功！"
        rm -f "$temp_cron" "${temp_cron}.clean"
    else
        echo "[$user] crontab 写入失败！"
        cat "$temp_cron"
        rm -f "$temp_cron" "${temp_cron}.clean"
        exit 1
    fi
}

# ---------------- 构造 crontab 任务（北京时间）----------------
CRON_JOB="$CRON_TIME TZ='$TZ' $SCRIPT_PATH >> $LOG_PATH 2>&1"

# ---------------- 执行添加 ----------------
add_cron_job "$CRON_JOB" "$CRON_USER"

# ---------------- 最终提示 ----------------
echo "==========================================="
echo "部署完成！全部使用北京时间 (CST)"
echo "脚本路径: $SCRIPT_PATH"
echo "日志路径: $LOG_PATH"
echo "阈值: 30 分钟"
echo "检查频率: 每 30 分钟"
echo "==========================================="
echo
echo "立即验证命令："
echo "  sudo crontab -l | grep k8s-restart-monitor"
echo "  sudo tail -f $LOG_PATH"
echo "  sudo bash $SCRIPT_PATH   # 手动运行一次"
echo
echo "当前北京时间: $(date '+%Y-%m-%d %H:%M:%S %Z')"